<!DOCTYPE html>

<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>cup.storage.obj &#8212; cup 3.2.33 documentation</title>
    <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
    <link rel="stylesheet" type="text/css" href="../../../_static/alabaster.css" />
    <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
    <script src="../../../_static/doctools.js"></script>
    <script src="../../../_static/sphinx_highlight.js"></script>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" />
   
  <link rel="stylesheet" href="../../../_static/custom.css" type="text/css" />
  
  
  <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />

  </head><body>
  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          

          <div class="body" role="main">
            
  <h1>Source code for cup.storage.obj</h1><div class="highlight"><pre>
<span></span><span class="ch">#!/usr/bin/env python</span>
<span class="c1"># -*- coding: utf-8 -*</span>
<span class="c1"># Copyright: [CUP] - See LICENSE for details.</span>
<span class="c1"># Authors: Guannan Ma (@mythmgn),</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">:description:</span>
<span class="sd">    Object related storage</span>
<span class="sd">&quot;&quot;&quot;</span>

<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">abc</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">shutil</span>
<span class="kn">import</span> <span class="nn">ftplib</span>
<span class="kn">import</span> <span class="nn">traceback</span>
<span class="kn">import</span> <span class="nn">logging</span>

<span class="kn">from</span> <span class="nn">cup</span> <span class="kn">import</span> <span class="n">log</span>
<span class="kn">from</span> <span class="nn">cup</span> <span class="kn">import</span> <span class="n">err</span>


<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
    <span class="s1">&#39;AFSObjectSystem&#39;</span><span class="p">,</span> <span class="s1">&#39;S3ObjectSystem&#39;</span><span class="p">,</span> <span class="s1">&#39;FTPObjectSystem&#39;</span><span class="p">,</span>
    <span class="s1">&#39;LocalObjectSystem&#39;</span>
<span class="p">]</span>


<span class="k">class</span> <span class="nc">ObjectInterface</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    object interface, abstract class. Should not be used directly</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">abc</span><span class="o">.</span><span class="n">ABCMeta</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param config:</span>
<span class="sd">            dict like config, should contains at leat</span>
<span class="sd">            {</span>
<span class="sd">                &#39;uri&#39;: &#39;xxxx&#39;,</span>
<span class="sd">                &#39;user&#39;: &#39;xxxx&#39;,   # or stands for accesskey</span>
<span class="sd">                &#39;passwords&#39;: &#39;xxxx&#39;, # or stands for secretkey</span>
<span class="sd">                &#39;extra&#39;: some_object</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_config</span> <span class="o">=</span> <span class="n">config</span>

    <span class="k">def</span> <span class="nf">_validate_config</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">,</span> <span class="n">keys</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;validate config if there&#39;s any missing items&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">keys</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">config</span><span class="p">:</span>
                <span class="n">ret</span> <span class="o">=</span> <span class="kc">False</span>

        <span class="k">return</span> <span class="n">ret</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">localfile</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param dest:</span>
<span class="sd">            system path</span>
<span class="sd">        :param localfile:</span>
<span class="sd">            localfile</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        delete a file</span>

<span class="sd">        :param path:</span>
<span class="sd">            object path</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object into localpath</span>
<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object info</span>
<span class="sd">        :return:</span>
<span class="sd">           {</span>
<span class="sd">               &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">               &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">               &#39;objectinfo&#39;: {</span>
<span class="sd">                   &#39;size&#39;: 1024, # at least have this one</span>
<span class="sd">                   &#39;atime&#39;: &#39;xxxx.xx.xx&#39;, # optional</span>
<span class="sd">                   &#39;mtime&#39;: &#39;xxxx.xx.xx&#39;, # optional</span>
<span class="sd">                   &#39;ctime&#39;: &#39;xxxx.xx.xx&#39;, # optional</span>
<span class="sd">                   .......</span>
<span class="sd">               }</span>
<span class="sd">           }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        mkdir dir of a path</span>
<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">rmdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rmdir of a path</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>

    <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rename from path to path&quot;&quot;&quot;</span>


<div class="viewcode-block" id="AFSObjectSystem"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem">[docs]</a><span class="k">class</span> <span class="nc">AFSObjectSystem</span><span class="p">(</span><span class="n">ObjectInterface</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    AFSObjectSystem implemented interface.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param config:</span>
<span class="sd">            be complied with cup.util.conf.Configure2Dict().get_dict().</span>
<span class="sd">            Shoule be dict like object</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ObjectInterface</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>

<div class="viewcode-block" id="AFSObjectSystem.put"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.put">[docs]</a>    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">localfile</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param dest:</span>
<span class="sd">            system path</span>
<span class="sd">        :param localfile:</span>
<span class="sd">            localfile</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>

<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.delete"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.delete">[docs]</a>    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        delete a file</span>

<span class="sd">        :param path:</span>
<span class="sd">            object path</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.get"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.get">[docs]</a>    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object into localpath</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.head"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.head">[docs]</a>    <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object info</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                    &#39;objectinfo&#39;: {</span>
<span class="sd">                    size: 1024,</span>
<span class="sd">                    .......</span>
<span class="sd">                    }</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.mkdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.mkdir">[docs]</a>    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        mkdir dir of a path</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                    &#39;objectinfo&#39;: {</span>
<span class="sd">                    size: 1024,</span>
<span class="sd">                    .......</span>
<span class="sd">                    }</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.rmdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.rmdir">[docs]</a>    <span class="k">def</span> <span class="nf">rmdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rmdir of a path&quot;&quot;&quot;</span></div>

<div class="viewcode-block" id="AFSObjectSystem.rename"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.AFSObjectSystem.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rename&quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">NotImplementedYet</span><span class="p">(</span><span class="s1">&#39;AFSObjectSystem.rename&#39;</span><span class="p">)</span></div></div>


<span class="c1"># pylint: disable=R0902</span>
<span class="c1"># need to have so many</span>
<div class="viewcode-block" id="S3ObjectSystem"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem">[docs]</a><span class="k">class</span> <span class="nc">S3ObjectSystem</span><span class="p">(</span><span class="n">ObjectInterface</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    s3 object system</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param config:</span>
<span class="sd">            be complied with cup.util.conf.Configure2Dict().get_dict().</span>
<span class="sd">            Shoule be dict like object</span>

<span class="sd">        :raise:</span>
<span class="sd">            cup.err.ConfigError if there&#39;s any config item missing</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ObjectInterface</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
        <span class="n">required_keys</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;ak&#39;</span><span class="p">,</span> <span class="s1">&#39;sk&#39;</span><span class="p">,</span> <span class="s1">&#39;endpoint&#39;</span><span class="p">,</span> <span class="s1">&#39;bucket&#39;</span><span class="p">]</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_validate_config</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">,</span> <span class="n">required_keys</span><span class="p">):</span>
            <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">ConfigError</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">required_keys</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ak</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;ak&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_sk</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;sk&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_endpoint</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;endpoint&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;bucket&#39;</span><span class="p">]</span>
        <span class="kn">import</span> <span class="nn">boto3</span>
        <span class="kn">from</span> <span class="nn">botocore</span> <span class="kn">import</span> <span class="n">exceptions</span>
        <span class="kn">from</span> <span class="nn">botocore</span> <span class="kn">import</span> <span class="n">client</span> <span class="k">as</span> <span class="n">coreclient</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_s3_config</span> <span class="o">=</span> <span class="n">coreclient</span><span class="o">.</span><span class="n">Config</span><span class="p">(</span>
            <span class="n">signature_version</span><span class="o">=</span><span class="s1">&#39;s3v4&#39;</span><span class="p">,</span>
            <span class="n">s3</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;addressing_style&#39;</span><span class="p">:</span> <span class="s1">&#39;path&#39;</span><span class="p">}</span>
        <span class="p">)</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;boto3&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;botocore&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
        <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;s3transfer&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to connect to boto3&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">client</span><span class="p">(</span>
            <span class="s1">&#39;s3&#39;</span><span class="p">,</span>
            <span class="n">aws_access_key_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_ak</span><span class="p">,</span>
            <span class="n">aws_secret_access_key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_sk</span><span class="p">,</span>
            <span class="n">endpoint_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_endpoint</span><span class="p">,</span>
            <span class="c1"># region_name=conf_dict[&#39;region_name&#39;],</span>
            <span class="n">config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_s3_config</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="o">=</span> <span class="n">exceptions</span><span class="o">.</span><span class="n">ClientError</span>

<div class="viewcode-block" id="S3ObjectSystem.put"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.put">[docs]</a>    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">localfile</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param dest:</span>
<span class="sd">            system path</span>
<span class="sd">        :param localfile:</span>
<span class="sd">            localfile</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to put object&#39;</span>
        <span class="p">}</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">localfile</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fhandle</span><span class="p">:</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">put_object</span><span class="p">(</span>
                    <span class="n">Key</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dest</span><span class="p">),</span>
                    <span class="n">Bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span><span class="p">,</span>
                    <span class="n">Body</span><span class="o">=</span><span class="n">fhandle</span>
                <span class="p">)</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
            <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.delete"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.delete">[docs]</a>    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        delete a file</span>

<span class="sd">        :param path:</span>
<span class="sd">            object path</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>

<span class="sd">            }</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">delete_object</span><span class="p">(</span>
                <span class="n">Key</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">),</span>
                <span class="n">Bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span>
            <span class="p">)</span>
        <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.get"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.get">[docs]</a>    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object into localpath</span>

<span class="sd">        :return:</span>
<span class="sd">            {</span>
<span class="sd">                &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                &#39;msg&#39;: &#39;if any&#39;</span>

<span class="sd">            }</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">localpath</span><span class="p">,</span> <span class="s1">&#39;w+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fhandle</span><span class="p">:</span>
                <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">get_object</span><span class="p">(</span>
                    <span class="n">Key</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">),</span>
                    <span class="n">Bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span>
                <span class="p">)</span>
                <span class="n">fhandle</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">resp</span><span class="p">[</span><span class="s1">&#39;Body&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.head"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.head">[docs]</a>    <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the object info</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                    &#39;objectinfo&#39;: {</span>
<span class="sd">                        size: 1024,</span>
<span class="sd">                        .......</span>
<span class="sd">                    }</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to get objectinfo&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span>
                <span class="n">Key</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">),</span>
                <span class="n">Bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_bucket</span>
            <span class="p">)</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;objectinfo&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">resp</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
        <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.mkdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.mkdir">[docs]</a>    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        mkdir dir of a path</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;,</span>
<span class="sd">                    &#39;objectinfo&#39;: {</span>
<span class="sd">                        size: 1024,</span>
<span class="sd">                        ..</span>
<span class="sd">                    }</span>
<span class="sd">                }</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">NotImplementedYet</span><span class="p">(</span><span class="s1">&#39;mkdir not supported for S3ObjectSystem&#39;</span><span class="p">)</span></div>

<div class="viewcode-block" id="S3ObjectSystem.rmdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.rmdir">[docs]</a>    <span class="k">def</span> <span class="nf">rmdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rmdir of a path&quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">NotImplementedYet</span><span class="p">(</span><span class="s1">&#39;rmdir not supported for S3ObjectSystem&#39;</span><span class="p">)</span></div>

<div class="viewcode-block" id="S3ObjectSystem.create_bucket"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.create_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">create_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;create bucket&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to create bucket&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">create_bucket</span><span class="p">(</span>
                <span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span>
            <span class="p">)</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
        <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.head_bucket"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.head_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">head_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;create bucket&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to create bucket&#39;</span><span class="p">,</span>
            <span class="s1">&#39;bucket_info&#39;</span><span class="p">:</span> <span class="kc">None</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span>
                <span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span>
            <span class="p">)</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;bucket_info&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">resp</span>
        <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.delete_bucket"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.delete_bucket">[docs]</a>    <span class="k">def</span> <span class="nf">delete_bucket</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket_name</span><span class="p">,</span> <span class="n">forcely</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;delete bucket</span>

<span class="sd">        :param forcely:</span>
<span class="sd">            if forcely is True, the bucket will be delete no matter it has</span>
<span class="sd">                objects inside. Otherwise, you have to delete items inside,</span>
<span class="sd">                then delete the bucket</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to create bucket&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">forcely</span><span class="p">:</span>
                <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_bucket</span><span class="p">(</span><span class="n">bucket_name</span><span class="p">)</span>
                <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">list_objects</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">)</span>
                <span class="k">if</span> <span class="s1">&#39;Contents&#39;</span> <span class="ow">in</span> <span class="n">res</span><span class="p">:</span>
                    <span class="k">for</span> <span class="n">obj</span> <span class="ow">in</span> <span class="n">res</span><span class="p">[</span><span class="s1">&#39;Contents&#39;</span><span class="p">]:</span>
                        <span class="k">try</span><span class="p">:</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">delete_object</span><span class="p">(</span>
                                <span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span><span class="p">,</span>
                                <span class="n">Key</span><span class="o">=</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">]</span>
                            <span class="p">)</span>
                        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
                            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;faield to delete obj in bucket&#39;</span>
                            <span class="k">return</span> <span class="n">ret</span>
            <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__s3conn</span><span class="o">.</span><span class="n">delete_bucket</span><span class="p">(</span>
                <span class="n">Bucket</span><span class="o">=</span><span class="n">bucket_name</span>
            <span class="p">)</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
        <span class="k">except</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="S3ObjectSystem.rename"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.S3ObjectSystem.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rename&quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">NotImplementedYet</span><span class="p">(</span><span class="s1">&#39;AFSObjectSystem.rename&#39;</span><span class="p">)</span></div></div>


<div class="viewcode-block" id="FTPObjectSystem"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem">[docs]</a><span class="k">class</span> <span class="nc">FTPObjectSystem</span><span class="p">(</span><span class="n">ObjectInterface</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    ftp object system, Plz notice all methods of FTPObjectSystem is NOT</span>
<span class="sd">    thread-safe! Be careful when you use it in a service of concurrency.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param config:</span>
<span class="sd">            {</span>
<span class="sd">                &quot;uri&quot;:&quot;ftp://host:port&quot;,</span>
<span class="sd">                &quot;user&quot;:&quot;username&quot;,</span>
<span class="sd">                &quot;password&quot;:&quot;password&quot;,</span>
<span class="sd">                &quot;extra&quot;:None   //timeout:30s</span>
<span class="sd">            }</span>

<span class="sd">        :raise:</span>
<span class="sd">            cup.err.ConfigError if there&#39;s any config item missing</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ObjectInterface</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
        <span class="n">required_keys</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;uri&#39;</span><span class="p">,</span> <span class="s1">&#39;user&#39;</span><span class="p">,</span> <span class="s1">&#39;passwords&#39;</span><span class="p">]</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_validate_config</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">,</span> <span class="n">required_keys</span><span class="p">):</span>
            <span class="k">raise</span> <span class="n">err</span><span class="o">.</span><span class="n">ConfigError</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">required_keys</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;uri&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_user</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;user&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_passwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;passwords&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;extra&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_dufault_timeout</span> <span class="o">=</span> <span class="mi">30</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_config</span><span class="p">[</span><span class="s1">&#39;extra&#39;</span><span class="p">],</span> <span class="nb">int</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_dufault_timeout</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to connect to ftp server&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_host</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_uri</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">][</span><span class="mi">2</span><span class="p">:]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_port</span> <span class="o">=</span> <span class="n">ftplib</span><span class="o">.</span><span class="n">FTP_PORT</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_uri</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)[</span><span class="mi">2</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_uri</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)[</span><span class="mi">2</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_port</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dufault_timeout</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_user</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_passwd</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_last_optime</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span> <span class="o">=</span> <span class="mi">15</span>  <span class="c1"># idle time for ftp</span>

    <span class="k">def</span> <span class="fm">__del__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;release connect&quot;&quot;&quot;</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span>
        <span class="k">except</span><span class="p">:</span>
            <span class="k">pass</span>

    <span class="k">def</span> <span class="nf">_check_timeout</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;check if we need to reconnect&quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_last_optime</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span><span class="p">:</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span>
            <span class="k">except</span><span class="p">:</span>
                <span class="k">pass</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_port</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dufault_timeout</span>
            <span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_user</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_passwd</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_last_optime</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">_get_relative_path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;get relative path for real actions&quot;&quot;&quot;</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
            <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">cwd</span><span class="p">):]</span>
        <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">path</span>

<div class="viewcode-block" id="FTPObjectSystem.put"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.put">[docs]</a>    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">destfile</span><span class="p">,</span> <span class="n">localfile</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        :param destfile:</span>
<span class="sd">            ftp path for the localfile</span>
<span class="sd">        :param localfile:</span>
<span class="sd">            localfile</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to put localfile </span><span class="si">{0}</span><span class="s1"> to ftp </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">localfile</span><span class="p">,</span> <span class="n">destfile</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">destdir</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">destfile</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="n">destfile</span><span class="p">)</span>
        <span class="n">destfile</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">destfile</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="n">rindex</span> <span class="o">=</span> <span class="n">destfile</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">rindex</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">destdir</span> <span class="o">=</span> <span class="n">cwd</span>
            <span class="n">file_name</span> <span class="o">=</span> <span class="n">destfile</span>
        <span class="k">elif</span> <span class="n">rindex</span> <span class="o">&gt;=</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">destfile</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;value error, destfile </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">destfile</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">destdir</span> <span class="o">=</span> <span class="n">destfile</span><span class="p">[:</span><span class="n">rindex</span><span class="p">]</span>
            <span class="n">file_name</span> <span class="o">=</span> <span class="n">destfile</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;put localfile </span><span class="si">{0}</span><span class="s1"> into ftp </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">localfile</span><span class="p">,</span> <span class="n">destfile</span><span class="p">))</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">localfile</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fhandle</span><span class="p">:</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">destdir</span><span class="p">)</span>
                <span class="n">ftp_cmd</span> <span class="o">=</span> <span class="s1">&#39;STOR </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">storbinary</span><span class="p">(</span><span class="n">ftp_cmd</span><span class="p">,</span> <span class="n">fhandle</span><span class="p">)</span>
            <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
                <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to put, err:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.delete"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.delete">[docs]</a>    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;delete file&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to delete ftp file: </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">())</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.get"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.get">[docs]</a>    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get a file into localpath</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to get ftp file </span><span class="si">{0}</span><span class="s1"> to  </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">localpath</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
            <span class="n">localpath</span> <span class="o">+=</span> <span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;to get ftp </span><span class="si">{0}</span><span class="s1"> to local </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">))</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">localpath</span><span class="p">,</span> <span class="s1">&#39;w+&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fhandle</span><span class="p">:</span>
                <span class="n">ftp_cmd</span> <span class="o">=</span> <span class="s1">&#39;RETR </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
                <span class="n">resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="n">ftp_cmd</span><span class="p">,</span> <span class="n">fhandle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to get </span><span class="si">{0}</span><span class="s1"> to </span><span class="si">{1}</span><span class="s1">, err:</span><span class="si">{2}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">,</span> <span class="n">error</span>
            <span class="p">)</span>
            <span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">])</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.head"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.head">[docs]</a>    <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the file info</span>

<span class="sd">        :return:</span>
<span class="sd">            ::</span>

<span class="sd">                {</span>
<span class="sd">                    &#39;returncode&#39;: 0 for success, others for failure,</span>
<span class="sd">                    &#39;msg&#39;: &#39;if any&#39;</span>
<span class="sd">                    &#39;fileinfo&#39;: [</span>
<span class="sd">                        &quot;-rw-rw-r-- 1 work work   201 Nov  9 17:03 __init__.py&quot;</span>
<span class="sd">                    ]</span>
<span class="sd">                }</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;failed to get objectinfo&#39;</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="n">res_info</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">f_flag</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">def</span> <span class="nf">_call_back</span><span class="p">(</span><span class="n">arg</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">f_flag</span> <span class="ow">and</span> <span class="n">arg</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="o">==</span> <span class="n">file_name</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">res_info</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">arg</span><span class="p">)</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">f_flag</span><span class="p">:</span>
                <span class="n">res_info</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">arg</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_file</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
                <span class="n">file_name</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="n">path</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:]</span>
                <span class="n">f_flag</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="n">pos</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
                <span class="n">p_path</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span> <span class="n">pos</span><span class="p">]</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">p_path</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">retrlines</span><span class="p">(</span><span class="s1">&#39;LIST&#39;</span><span class="p">,</span> <span class="n">_call_back</span><span class="p">)</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;fileinfo&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">res_info</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;success&#39;</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.mkdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.mkdir">[docs]</a>    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        mkdir</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">recursive</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">mkd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">subdirs</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">subdir</span> <span class="ow">in</span> <span class="n">subdirs</span><span class="p">:</span>
                    <span class="k">try</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">subdir</span><span class="p">)</span>
                    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">mkd</span><span class="p">(</span><span class="n">subdir</span><span class="p">)</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">subdir</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to mkdir, err:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.rmdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.rmdir">[docs]</a>    <span class="k">def</span> <span class="nf">rmdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        rmdir</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">recursive</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">rmd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">cwd</span> <span class="o">=</span>  <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
                <span class="n">allItems</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">nlst</span><span class="p">()</span>
                <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">allItems</span><span class="p">:</span>
                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_file</span><span class="p">(</span><span class="n">item</span><span class="p">):</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
                    <span class="k">else</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">rmdir</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">rmd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to rmdir, err:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="FTPObjectSystem.is_file"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.is_file">[docs]</a>    <span class="k">def</span> <span class="nf">is_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;path is file or not&quot;&quot;&quot;</span>
        <span class="n">res</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_timeout</span><span class="p">()</span>
        <span class="n">cwd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">pwd</span><span class="p">()</span>
        <span class="n">path</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_relative_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">cwd</span><span class="p">)</span>
        <span class="n">res_info</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">def</span> <span class="nf">_call_back</span><span class="p">(</span><span class="n">arg</span><span class="p">):</span>
            <span class="n">res_info</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">arg</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">res</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
            <span class="k">pass</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">pos</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">pos</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
                <span class="n">file_name</span> <span class="o">=</span> <span class="n">path</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">p_path</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span> <span class="n">pos</span><span class="p">]</span>
                <span class="n">file_name</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="n">pos</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:]</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">p_path</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">retrlines</span><span class="p">(</span><span class="s1">&#39;MLSD&#39;</span><span class="p">,</span> <span class="n">_call_back</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">res_info</span><span class="p">:</span>
                <span class="k">if</span> <span class="n">item</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;;&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="o">==</span> <span class="n">file_name</span> <span class="ow">and</span> <span class="s1">&#39;type=file&#39;</span> <span class="ow">in</span> <span class="n">item</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
                    <span class="k">return</span> <span class="kc">True</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">cwd</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="k">pass</span>
        <span class="k">return</span> <span class="kc">False</span></div>

<div class="viewcode-block" id="FTPObjectSystem.rename"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.FTPObjectSystem.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rename frompath to path&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_ftp_con</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to rename from </span><span class="si">{0}</span><span class="s1"> to </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div></div>


<div class="viewcode-block" id="LocalObjectSystem"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem">[docs]</a><span class="k">class</span> <span class="nc">LocalObjectSystem</span><span class="p">(</span><span class="n">ObjectInterface</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;local object system&quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">kvconfig</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        initialize</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">config</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;uri&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
            <span class="s1">&#39;user&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>   <span class="c1"># or stands for accesskey</span>
            <span class="s1">&#39;passwords&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span> <span class="c1"># or stands for secretkey</span>
            <span class="s1">&#39;extra&#39;</span><span class="p">:</span> <span class="kc">None</span>
        <span class="p">}</span>
        <span class="n">ObjectInterface</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>

<div class="viewcode-block" id="LocalObjectSystem.put"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.put">[docs]</a>    <span class="k">def</span> <span class="nf">put</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">localfile</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        local object put == copy</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">shutil</span><span class="o">.</span><span class="n">copy2</span><span class="p">(</span><span class="n">dest</span><span class="p">,</span> <span class="n">localfile</span><span class="p">)</span>
        <span class="c1"># pylint: disable=W0703</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to put:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="LocalObjectSystem.delete"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.delete">[docs]</a>    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;delete a file in local&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="c1"># pylint: disable=W0703</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to unlink file:</span><span class="si">{0}</span><span class="s1">, err:</span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">path</span><span class="p">,</span> <span class="n">error</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="LocalObjectSystem.get"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.get">[docs]</a>    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get a file into localpath</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">localpath</span><span class="p">)</span></div>

<div class="viewcode-block" id="LocalObjectSystem.head"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.head">[docs]</a>    <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;get the object info&quot;&quot;&quot;</span>
        <span class="n">retcode</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="n">msg</span> <span class="o">=</span> <span class="s1">&#39;ok&#39;</span>
        <span class="n">objectinfo</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
            <span class="n">retcode</span> <span class="o">=</span> <span class="mi">255</span>
            <span class="n">msg</span> <span class="o">=</span> <span class="s1">&#39;file/dir not found&#39;</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">statinfo</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
            <span class="n">objectinfo</span> <span class="o">=</span>  <span class="p">{</span>
                <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="n">statinfo</span><span class="o">.</span><span class="n">st_size</span><span class="p">,</span>
                <span class="s1">&#39;atime&#39;</span><span class="p">:</span> <span class="n">statinfo</span><span class="o">.</span><span class="n">st_atime</span><span class="p">,</span>
                <span class="s1">&#39;mtime&#39;</span><span class="p">:</span> <span class="n">statinfo</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">,</span>
                <span class="s1">&#39;ctime&#39;</span><span class="p">:</span> <span class="n">statinfo</span><span class="o">.</span><span class="n">st_ctime</span>
            <span class="p">}</span>
        <span class="n">info_dict</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="n">retcode</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="n">msg</span><span class="p">,</span>
            <span class="s1">&#39;objectinfo&#39;</span><span class="p">:</span> <span class="n">objectinfo</span>
        <span class="p">}</span>
        <span class="k">return</span> <span class="n">info_dict</span></div>

<div class="viewcode-block" id="LocalObjectSystem.mkdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.mkdir">[docs]</a>    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        mkdir</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="n">func</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">recursive</span><span class="p">:</span>
            <span class="n">func</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">func</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">IOError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to mkdir, err:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="LocalObjectSystem.rmdir"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.rmdir">[docs]</a>    <span class="k">def</span> <span class="nf">rmdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        rmdir</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="n">func</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">rmdir</span>
        <span class="k">if</span> <span class="n">recursive</span><span class="p">:</span>
            <span class="n">func</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">func</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">IOError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to rmdir, err:</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="LocalObjectSystem.rename"><a class="viewcode-back" href="../../../cup.storage.html#cup.storage.obj.LocalObjectSystem.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;rename from path to path&quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;returncode&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="s1">&#39;msg&#39;</span><span class="p">:</span> <span class="s1">&#39;success&#39;</span>
        <span class="p">}</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">os</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">IOError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;returncode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
            <span class="n">ret</span><span class="p">[</span><span class="s1">&#39;msg&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;failed to rename </span><span class="si">{0}</span><span class="s1"> to </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">frompath</span><span class="p">,</span> <span class="n">topath</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="n">ret</span></div></div>

<span class="c1"># vi:set tw=0 ts=4 sw=4 nowrap fdm=indent</span>
</pre></div>

          </div>
          
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
        <div class="sphinxsidebarwrapper">
<h1 class="logo"><a href="../../../index.html">cup</a></h1>








<h3>Navigation</h3>
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../cup.html">cup package</a></li>
</ul>

<div class="relations">
<h3>Related Topics</h3>
<ul>
  <li><a href="../../../index.html">Documentation overview</a><ul>
  <li><a href="../../index.html">Module code</a><ul>
  </ul></li>
  </ul></li>
</ul>
</div>
<div id="searchbox" style="display: none" role="search">
  <h3 id="searchlabel">Quick search</h3>
    <div class="searchformwrapper">
    <form class="search" action="../../../search.html" method="get">
      <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
      <input type="submit" value="Go" />
    </form>
    </div>
</div>
<script>document.getElementById('searchbox').style.display = "block"</script>








        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="footer">
      &copy;2023, CUP-DEV Team.
      
      |
      Powered by <a href="http://sphinx-doc.org/">Sphinx 7.0.1</a>
      &amp; <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.13</a>
      
    </div>

    

    
  </body>
</html>